import requests
from lxml import etree
import json


class XiuShi(object):
    """抓取糗事百科"""
    def __init__(self):
        self.url = 'https://www.qiushibaike.com/8hr/page/{}'
        self.headers = {
            'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'
        }
        self.file = open('qiushi.json', 'w')

    def url_list(self):
        print('正在生成url列表')
        urls = [self.url.format(i) for i in range(1, 14)]
        # print(urls)
        return urls

    def get_data(self, url):
        print('正在发送请求')
        response = requests.get(url, headers=self.headers)
        # print(response.content)
        return response.content

    def parse_page(self, data):
        print('正在解析数据')
        # 生成xpath对象
        html = etree.HTML(data)
        # 解析所有帖子的结点
        node_list = html.xpath('//*[contains(@id,"qiushi_tag_")]')
        qiushi_list = []
        for node in node_list:
            qiu_dict = dict()
            try:
                qiu_dict['user'] = node.xpath('./div[1]/a[2]/h2/text()')[0].strip()
                qiu_dict['age'] = node.xpath('./div[1]/div/text()')[0]
                qiu_dict['url'] = 'https://www.qiushibaike.com' + node.xpath('./div[1]/a[1]/@href')[0]
                qiu_dict['gender'] = node.xpath('./div[1]/div/@class')[0].split(' ')[-1]
            except:
                qiu_dict['user'] = '匿名用户'
                qiu_dict['age'] = None
                qiu_dict['url'] = None
                qiu_dict['gender'] = None
            qiu_dict['text'] = node.xpath('./a/div/span/text()')[0].strip()
            qiushi_list.append(qiu_dict)

        return qiushi_list

    def save_data(self, qiushi_list):
        print('正在保存数据')
        for qiushi in qiushi_list:
            # print(qiushi)
            # 转化为json数据
            json_data = json.dumps(qiushi, ensure_ascii=False) + ',\n'
            print(json_data)
            self.file.write(json_data)

    def __del__(self):
        """关闭文件"""
        self.file.close()

    def run(self):
        urls = self.url_list()
        for url in urls:
            data = self.get_data(url)
            qiushi_list = self.parse_page(data)
            self.save_data(qiushi_list)




if __name__ == '__main__':
    qiu = XiuShi()
    qiu.run()